from spidertools.utils.pinyin_utils import pinyin
from spidertools.utils.time_utils import get_current_datetime
from spidertools.utils.mutiprocess_utils import parallel_apply
import pydoc

from utils.logger_utils import savelog
from configs.logging_settings import LogStateString
from project_setting import DEBUG
from pprint import pprint
import traceback



def make_some_extra_info(output_dict,origin_dict):
    output_dict["原始地址"] = origin_dict['origin_url']
    output_dict['公告类型'] = output_dict['工程公告信息']["公告类型"]
    output_dict['标段名称'] = output_dict['工程招标信息']["标段名称"]
    output_dict["工程公告信息发布时间"] = output_dict['工程公告信息']["工程公告信息发布时间"]
    output_dict["工程公告信息"]['工程来源网站'] = origin_dict['source_type']
    output_dict["is_parsed"] = 0
    output_dict['search_key'] = origin_dict['origin_url'] + "$$" + origin_dict['公告类型'] + "$$" + output_dict["工程公告信息发布时间"] + "$$" + output_dict["标段名称"]

def single_html_extact(info_dict):
    '''
    解析单个公告内容
    :param info_dict:
    :return:
    '''
    output = {"ignore":1}
    try:
        if "is_parsed" in info_dict  and info_dict['is_parsed'] != 0:
           return output
        module_list = ["extractors", 'commonresources']
        if 'province' in info_dict:
            info_dict['province'] = info_dict['province'].replace("省", "")#用于解决省会名称带着省的问题
            module_list.append(pinyin(info_dict['province']))
        #if 'city' in info_dict:
        #    module_list.append(pinyin(info_dict['city']))
        module_list.append(pinyin(info_dict['source_type']))
        module_list.append(pinyin(info_dict['source_type'],upper_first_char=True))

        full_class_name = ".".join(module_list)
        obj = pydoc.locate(full_class_name)(info_dict)
        output_result = obj.output_extractor_dict()


        if output_result:
            if type(output_result) == list:
                for item in output_result:
                    make_some_extra_info(item,info_dict)
            elif type(output_result) == dict:
                make_some_extra_info(output_result,info_dict)
            output = output_result



    except Exception as e:
        if "html" in info_dict:
            del info_dict['html']
        if "_id" in info_dict:
            del info_dict["_id"]
        savelog(LogStateString.FatalError,traceback.format_exc(),msg_dict=info_dict)
    return output


def parse_callback(insert_item,mongo_db,change_is_parsed=True):
    '''处理信息提取的结果'''
    try:
        mongo_extor_table = mongo_db['project_info_dict']
        mongo_html_table = mongo_db['project_html']
        if insert_item and 'ignore' not in insert_item:
            if type(insert_item) == list:
                for item in insert_item:
                    if change_is_parsed:
                        mongo_html_table.update({"origin_url": item['原始地址']}, {"$set": {"is_parsed":1}})
                    search_dict = {"search_key": item['search_key']}
                    search_result = mongo_extor_table.count_documents(search_dict)
                    if search_result:
                        item['update_time'] = get_current_datetime()
                        update_value = {"$set": item}
                        mongo_extor_table.update_one(search_dict, update_value)
                    else:
                        item['create_time'] = get_current_datetime()
                        item['update_time'] = get_current_datetime()
                        mongo_extor_table.insert_one(item)
            else:
                if change_is_parsed:
                    mongo_html_table.update({"origin_url": insert_item['原始地址']}, {"$set": {"is_parsed":1}})
                search_dict = {"search_key": insert_item['search_key']}
                search_result = mongo_extor_table.find(search_dict).count()
                if search_result:
                    insert_item['update_time'] = get_current_datetime()
                    update_value = {"$set": insert_item}
                    mongo_extor_table.update_one(search_dict, update_value)
                else:
                    insert_item['create_time'] = get_current_datetime()
                    insert_item['update_time'] = get_current_datetime()
                    mongo_extor_table.insert_one(insert_item)


        pprint(insert_item)
    except Exception as e:
        print(e)




def mutiproces_parse(search_condition,mongo_db,dummy=False,epoch_size=5000,process_num=8,change_is_parsed=False,max_epoch=0,begin_num=0):
    '''
        多线程或者多进程解析网页
    '''
    mongo_table = mongo_db['project_html']
    total_count = mongo_table.find(search_condition).count()
    print(total_count)
    for i in range(int(total_count/epoch_size) + 1):
        print("第"+str(i+1)+"次开始")
        skip_num = epoch_size * i + begin_num
        items =  mongo_table.find(search_condition).skip(skip_num).limit(epoch_size)
        queue_max_num = epoch_size * 10
        print("start")
        parallel_apply(
            func=single_html_extact,
            iterable=items,
            workers=process_num,
            max_queue_size=queue_max_num,
            callback=parse_callback,
            dummy=dummy,
            callback_params={"mongo_db":mongo_db,"change_is_parsed":change_is_parsed}
        )

        if max_epoch != 0:
            if i >= max_epoch:
                print("达到你设置的最大epoch:%s,退出检查 " % max_epoch)
                break